<!DOCTYPE html>
<html>
<head>
  <meta charset="utf-8">
  <title>spark3.0新特性 | iworkh blog</title>
  
    <meta name="keywords" content=" spark ">
  
  <meta name="description" content="spark3.0新特性 | iworkh blog">
    <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
<meta name="description" content="个人简介沐雨云楼，程序员一名。 喜欢研究技术，主要从事JAVA、微服务、大数据领域。 联系方式 QQ : 157162006 邮箱 :157162006@qq.com">
<meta property="og:type" content="website">
<meta property="og:title" content="关于">
<meta property="og:url" content="https://iworkh.gitee.io/blog/about/index.html">
<meta property="og:site_name" content="iworkh blog">
<meta property="og:description" content="个人简介沐雨云楼，程序员一名。 喜欢研究技术，主要从事JAVA、微服务、大数据领域。 联系方式 QQ : 157162006 邮箱 :157162006@qq.com">
<meta property="og:locale" content="zh_CN">
<meta property="article:published_time" content="2020-05-17T07:22:16.000Z">
<meta property="article:modified_time" content="2020-09-12T13:21:47.523Z">
<meta property="article:author" content="沐雨云楼">
<meta property="article:tag" content="iworkh">
<meta property="article:tag" content="沐雨云楼">
<meta property="article:tag" content="blog">
<meta name="twitter:card" content="summary">


<link rel="icon" href="/blog/img/iwork.svg">

<link href="/blog/css/style.css?v=1.0.1" rel="stylesheet">

<link href="/blog/css/hl_theme/atom-light.css?v=1.0.1" rel="stylesheet">

<link href="//cdn.bootcdn.net/ajax/libs/animate.css/4.1.0/animate.min.css" rel="stylesheet">
<link href="//cdn.bootcdn.net/ajax/libs/font-awesome/5.13.0/js/fontawesome.min.js" rel="stylesheet">

<script src="//cdn.bootcdn.net/ajax/libs/jquery/3.5.1/jquery.min.js"></script>
<script src="/blog/js/jquery.autocomplete.min.js?v=1.0.1" ></script>
<script src="/blog/js/titleTip.js?v=1.0.1" ></script>

<script src="//cdn.bootcdn.net/ajax/libs/highlight.js/10.0.3/highlight.min.js"></script>
<script>
    hljs.initHighlightingOnLoad();
</script>

<script src="//cdn.bootcdn.net/ajax/libs/nprogress/0.2.0/nprogress.min.js"></script>



<script src="//cdn.bootcdn.net/ajax/libs/jquery-cookie/1.4.1/jquery.cookie.min.js" ></script>

<script src="/blog/js/iconfont.js?v=1.0.1" ></script>

<meta name="generator" content="Hexo 4.2.1"></head>
<div style="display: none">
  <input class="theme_disqus_on" value="false">
  <input class="theme_preload_comment" value="false">
  <input class="theme_blog_path" value="/blog">
</div>




<body>
<aside class="nav">
    <div class="nav-left">
        <a href="/blog/" class="avatar_target">
    <img class="avatar" src="/blog/img/iwork.svg" />
</a>
<div class="author">
    <span>沐雨云楼</span>
</div>

<div class="icon">
    
        
        <a title="github" href="https://gitee.com/iworkh" target="_blank">
            
                <svg class="iconfont-svg" aria-hidden="true">
                    <use xlink:href="#icon-github"></use>
                </svg>
            
        </a>
        
    
        
        <a title="csdn" href="https://blog.csdn.net/u011622109" target="_blank">
            
                <svg class="iconfont-svg" aria-hidden="true">
                    <use xlink:href="#icon-csdn"></use>
                </svg>
            
        </a>
        
    
        
        <a title="email" href="mailto:157162006@qq.com" target="_blank">
            
                <svg class="iconfont-svg" aria-hidden="true">
                    <use xlink:href="#icon-email"></use>
                </svg>
            
        </a>
        
    
        
        <a title="qq" href="http://wpa.qq.com/msgrd?v=3&uin=157162006&site=qq&menu=yes" target="_blank">
            
                <svg class="iconfont-svg" aria-hidden="true">
                    <use xlink:href="#icon-qq"></use>
                </svg>
            
        </a>
        
    
        
        <a title="kugou" href="https://www.kugou.com/" target="_blank">
            
                <svg class="iconfont-svg" aria-hidden="true">
                    <use xlink:href="#icon-kugou"></use>
                </svg>
            
        </a>
        
    
        
        <a title="neteasemusic" href="https://music.163.com/" target="_blank">
            
                <svg class="iconfont-svg" aria-hidden="true">
                    <use xlink:href="#icon-neteasemusic"></use>
                </svg>
            
        </a>
        
    
</div>



<a class="more-menus">更多菜单</a>


<ul>
    <li><div class="all active">全部文章<small>(85)</small></div></li>
    
        
            
            <li><div data-rel="手册"><i class="fold iconfont icon-right"></i>手册<small>(2)</small></div>
                
                    <ul class="sub hide">
                        
                        <li><div data-rel="iworkh">iworkh<small>(1)</small></div>
                            
                        </li>
                            
                        <li><div data-rel="pgmanor">pgmanor<small>(1)</small></div>
                            
                        </li>
                            
                    </ul>
                
            </li>
            
        
    
        
            
            <li><div data-rel="工具"><i class="fold iconfont icon-right"></i>工具<small>(16)</small></div>
                
                    <ul class="sub hide">
                        
                        <li><div data-rel="git">git<small>(2)</small></div>
                            
                        </li>
                            
                        <li><div data-rel="github">github<small>(1)</small></div>
                            
                        </li>
                            
                        <li><div data-rel="hexo">hexo<small>(1)</small></div>
                            
                        </li>
                            
                        <li><div data-rel="idea">idea<small>(2)</small></div>
                            
                        </li>
                            
                        <li><div data-rel="markdowm">markdowm<small>(1)</small></div>
                            
                        </li>
                            
                        <li><div data-rel="maven">maven<small>(1)</small></div>
                            
                        </li>
                            
                        <li><div data-rel="有道云">有道云<small>(1)</small></div>
                            
                        </li>
                            
                    </ul>
                
            </li>
            
        
    
        
            
            <li><div data-rel="java"><i class="fold iconfont icon-right"></i>java<small>(20)</small></div>
                
                    <ul class="sub hide">
                        
                        <li><div data-rel="dao">dao<small>(6)</small></div>
                            
                        </li>
                            
                        <li><div data-rel="jvm">jvm<small>(1)</small></div>
                            
                        </li>
                            
                        <li><div data-rel="test">test<small>(5)</small></div>
                            
                        </li>
                            
                        <li><div data-rel="tools">tools<small>(3)</small></div>
                            
                        </li>
                            
                        <li><div data-rel="并发">并发<small>(1)</small></div>
                            
                        </li>
                            
                    </ul>
                
            </li>
            
        
    
        
            
            <li><div data-rel="微服务"><i class="fold iconfont icon-right"></i>微服务<small>(6)</small></div>
                
                    <ul class="sub hide">
                        
                        <li><div data-rel="spring">spring<small>(2)</small></div>
                            
                        </li>
                            
                        <li><div data-rel="springboot">springboot<small>(3)</small></div>
                            
                        </li>
                            
                    </ul>
                
            </li>
            
        
    
        
            
            <li><div data-rel="架构">架构<small>(2)</small></div>
                
            </li>
            
        
    
        
            
            <li><div data-rel="大数据"><i class="fold iconfont icon-right"></i>大数据<small>(4)</small></div>
                
                    <ul class="sub hide">
                        
                        <li><div data-rel="hadoop">hadoop<small>(1)</small></div>
                            
                        </li>
                            
                        <li><div data-rel="spark">spark<small>(3)</small></div>
                            
                        </li>
                            
                    </ul>
                
            </li>
            
        
    
        
            
            <li><div data-rel="web"><i class="fold iconfont icon-right"></i>web<small>(3)</small></div>
                
                    <ul class="sub hide">
                        
                        <li><div data-rel="vue">vue<small>(2)</small></div>
                            
                        </li>
                            
                    </ul>
                
            </li>
            
        
    
        
            
            <li><div data-rel="db"><i class="fold iconfont icon-right"></i>db<small>(4)</small></div>
                
                    <ul class="sub hide">
                        
                        <li><div data-rel="mysql">mysql<small>(3)</small></div>
                            
                        </li>
                            
                        <li><div data-rel="redis">redis<small>(1)</small></div>
                            
                        </li>
                            
                    </ul>
                
            </li>
            
        
    
        
            
            <li><div data-rel="ai">ai<small>(4)</small></div>
                
            </li>
            
        
    
        
            
            <li><div data-rel="python"><i class="fold iconfont icon-right"></i>python<small>(8)</small></div>
                
                    <ul class="sub hide">
                        
                        <li><div data-rel="tornado">tornado<small>(3)</small></div>
                            
                        </li>
                            
                    </ul>
                
            </li>
            
        
    
        
            
            <li><div data-rel="linux"><i class="fold iconfont icon-right"></i>linux<small>(2)</small></div>
                
                    <ul class="sub hide">
                        
                        <li><div data-rel="docker">docker<small>(1)</small></div>
                            
                        </li>
                            
                        <li><div data-rel="nginx">nginx<small>(1)</small></div>
                            
                        </li>
                            
                    </ul>
                
            </li>
            
        
    
        
            
            <li><div data-rel="windows">windows<small>(3)</small></div>
                
            </li>
            
        
    
        
            
            <li><div data-rel="结构和算法">结构和算法<small>(8)</small></div>
                
            </li>
            
        
    
        
            
            <li><div data-rel="面试">面试<small>(2)</small></div>
                
            </li>
            
        
    
        
            
            <li><div data-rel="其他">其他<small>(1)</small></div>
                
            </li>
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
        
            
        
    
</ul>
<div class="left-bottom">
    <div class="menus">
    
    
    
    <a class="dynamic-menu " target="_blank"  href="https://pgmanor.gitee.io/blog/">pgmanor</a>
    
    
    <a class="dynamic-menu " target="_blank"  href="https://www.iworkh.com/manualIt/Category/scopeDev/">iworkh</a>
    
    
    <a class="dynamic-menu " target="_blank"  href="https://gitee.com/iworkh">gitee</a>
    
    </div>
    <div><a class="about  hasFriend  site_url"  href="/blog/about">关于</a><a style="width: 50%"  class="friends">友链</a></div>
</div>
<input type="hidden" id="iworkh_site_posts_number" value="85">
<input type="hidden" id="iworkh_site_word_count" value="197.8k">
<div style="display: none">
    <span id="busuanzi_value_site_uv"></span>
    <span id="busuanzi_value_site_pv"></span>
</div>

    </div>
    <div class="nav-right">
        <div class="friends-area">
    <div class="friends-title">
        友情链接
        <i class="back-title-list"></i>
    </div>
    <div class="friends-content">
        <ul>
            
            <li><a target="_blank" href="https://www.iworkh.com">iworkh</a></li>
            
            <li><a target="_blank" href="https://www.iworkh.com/download/share/">下载地址</a></li>
            
            <li><a target="_blank" href="http://yelog.org/">叶落阁</a></li>
            
        </ul>
    </div>
</div>
        <div class="title-list">
    <form onkeydown="if(event.keyCode === 13){return false;}">
        <input id="local-search-input" class="search" type="text" placeholder="in: 开头全文搜索" />
        <i class="cross"></i>
        <span>
            <label for="tagswitch">T:</label>
            <input id="tagswitch" type="checkbox" style="display: none" />
            <i id="tagsWitchIcon"></i>
        </span>
    </form>
    <div class="tags-list">
    
    <li class="article-tag-list-item">
        <a class="color3">ai</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">attr</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color1">aysnc</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">beanmap</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color1">bigdecimal</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">blog</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color2">chrome</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">dict</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color4">dll</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color2">docker</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color2">domain</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color4">fastjson</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">func</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color4">git</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color2">github</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color2">hadoop</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">hexo</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">idea</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color2">iworkh</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">jackson</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">java</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">jdni</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color2">jfinalshell</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color2">joplin</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color4">JPA</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">lock</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color4">markdowm</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color1">maven</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">mock</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">mq</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color1">mycat</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color1">mysql</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color1">nginx</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color4">notebook</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">plan</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">powermock</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color2">python</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color1">redis</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">rest</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color4">sharding-jdbc</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color1">spark</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color1">springboot</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">test</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">tool</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color1">tools</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">tornado</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color4">transactional</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color4">web</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">windows</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">wtforms</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">zookeeper</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">分库分表</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">分页</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">命令</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">工具</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">并发</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color4">序列化</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color4">微服务</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">性能分析</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">手册</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color4">有道云</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">标签</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">模板</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color5">百度云盘</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color1">结构和算法</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">计划</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">迁移</a>
    </li>
    
    <li class="article-tag-list-item">
        <a class="color3">面试</a>
    </li>
    
    <div class="clearfix"></div>
</div>

    
    <div id="local-search-result">

    </div>
    
    <nav id="title-list-nav">
        
        <a id="top" class="手册 pgmanor "
           href="/blog/2020/12/15/redirect-to-pgmanor/"
           data-tag="迁移"
           data-author="沐雨云楼" >
            <span class="post-title" title="😘博客迁移">😘博客迁移</span>
            <span class="post-date" title="2020-12-15 23:47:55">2020/12/15</span>
        </a>
        
        <a id="top" class="手册 iworkh "
           href="/blog/2020/05/27/manual-iworkh/"
           data-tag="手册,iworkh"
           data-author="沐雨云楼" >
            <span class="post-title" title="iworkh所有手册">iworkh所有手册</span>
            <span class="post-date" title="2020-05-27 21:37:04">2020/05/27</span>
        </a>
        
        <a id="top" class="微服务 springboot "
           href="/blog/2020/06/27/spring_boot_all_resources/"
           data-tag="springboot"
           data-author="沐雨云楼" >
            <span class="post-title" title="Spring Boot全网优质教程汇总">Spring Boot全网优质教程汇总</span>
            <span class="post-date" title="2020-06-27 17:43:13">2020/06/27</span>
        </a>
        
        <a id="top" class="工具 "
           href="/blog/2020/06/22/baidu-pandisk-share/"
           data-tag="百度云盘"
           data-author="沐雨云楼" >
            <span class="post-title" title="百度云盘分享链接">百度云盘分享链接</span>
            <span class="post-date" title="2020-06-22 18:38:21">2020/06/22</span>
        </a>
        
        <a id="top" class="工具 "
           href="/blog/2020/06/21/tool_blogs_all/"
           data-tag="blog"
           data-author="沐雨云楼" >
            <span class="post-title" title="优质博客和文章">优质博客和文章</span>
            <span class="post-date" title="2020-06-21 20:02:14">2020/06/21</span>
        </a>
        
        <a id="top" class="java jvm "
           href="/blog/2020/06/04/java-online-analysis/"
           data-tag="性能分析"
           data-author="沐雨云楼" >
            <span class="post-title" title="JAVA线上故障排查全套路">JAVA线上故障排查全套路</span>
            <span class="post-date" title="2020-06-04 20:34:05">2020/06/04</span>
        </a>
        
        <a id="top" class="工具 idea "
           href="/blog/2020/05/31/jetbrains-idea/"
           data-tag="工具,idea"
           data-author="沐雨云楼" >
            <span class="post-title" title="idea使用技巧">idea使用技巧</span>
            <span class="post-date" title="2020-05-31 16:12:30">2020/05/31</span>
        </a>
        
        <a id="top" class="工具 有道云 "
           href="/blog/2020/05/18/youdao_config_edit/"
           data-tag="工具,有道云"
           data-author="沐雨云楼" >
            <span class="post-title" title="有道云配置修改">有道云配置修改</span>
            <span class="post-date" title="2020-05-18 21:37:04">2020/05/18</span>
        </a>
        
        <a id="top" class="web "
           href="/blog/2020/06/01/web-tools/"
           data-tag="web,tool"
           data-author="沐雨云楼" >
            <span class="post-title" title="前端常用工具">前端常用工具</span>
            <span class="post-date" title="2020-06-01 21:37:04">2020/06/01</span>
        </a>
        
        <a  class="工具 "
           href="/blog/2020/10/25/tool-linux-jfinalshell/"
           data-tag="jfinalshell"
           data-author="沐雨云楼" >
            <span class="post-title" title="linux连接工具jfinalshell">linux连接工具jfinalshell</span>
            <span class="post-date" title="2020-10-25 10:24:59">2020/10/25</span>
        </a>
        
        <a  class="微服务 springboot "
           href="/blog/2020/10/06/spring-boot-multi-profiles/"
           data-tag="springboot"
           data-author="沐雨云楼" >
            <span class="post-title" title="springboot多环境配置打包">springboot多环境配置打包</span>
            <span class="post-date" title="2020-10-06 21:36:42">2020/10/06</span>
        </a>
        
        <a  class="web vue "
           href="/blog/2020/10/06/vue_springboot_package_publish/"
           data-tag="maven"
           data-author="沐雨云楼" >
            <span class="post-title" title="vue+springboot打包发布">vue+springboot打包发布</span>
            <span class="post-date" title="2020-10-06 14:22:29">2020/10/06</span>
        </a>
        
        <a  class="java dao "
           href="/blog/2020/10/06/java-jpa-query/"
           data-tag="JPA"
           data-author="沐雨云楼" >
            <span class="post-title" title="JPA自定义sql">JPA自定义sql</span>
            <span class="post-date" title="2020-10-06 10:20:38">2020/10/06</span>
        </a>
        
        <a  class="微服务 spring "
           href="/blog/2020/10/03/spring-pageable/"
           data-tag="分页"
           data-author="沐雨云楼" >
            <span class="post-title" title="vue+springboot分页交互">vue+springboot分页交互</span>
            <span class="post-date" title="2020-10-03 11:08:21">2020/10/03</span>
        </a>
        
        <a  class="工具 "
           href="/blog/2020/10/02/chrome-plugin/"
           data-tag="chrome"
           data-author="沐雨云楼" >
            <span class="post-title" title="chrome插件">chrome插件</span>
            <span class="post-date" title="2020-10-02 10:11:31">2020/10/02</span>
        </a>
        
        <a  class="windows "
           href="/blog/2020/10/01/windows-less-dll/"
           data-tag="dll"
           data-author="沐雨云楼" >
            <span class="post-title" title="windows缺少dll">windows缺少dll</span>
            <span class="post-date" title="2020-10-01 20:22:28">2020/10/01</span>
        </a>
        
        <a  class="java "
           href="/blog/2020/09/13/java-base-bigdecimal/"
           data-tag="bigdecimal"
           data-author="沐雨云楼" >
            <span class="post-title" title="BigDecimal正确使用了吗?">BigDecimal正确使用了吗?</span>
            <span class="post-date" title="2020-09-13 10:28:54">2020/09/13</span>
        </a>
        
        <a  class="windows "
           href="/blog/2020/09/12/windows-dev-install/"
           data-tag="windows"
           data-author="沐雨云楼" >
            <span class="post-title" title="windows开发环境安装">windows开发环境安装</span>
            <span class="post-date" title="2020-09-12 21:39:01">2020/09/12</span>
        </a>
        
        <a  class="工具 "
           href="/blog/2020/09/06/tool-notebook-joplin/"
           data-tag="joplin"
           data-author="沐雨云楼" >
            <span class="post-title" title="joplin笔记">joplin笔记</span>
            <span class="post-date" title="2020-09-06 20:50:55">2020/09/06</span>
        </a>
        
        <a  class="web vue "
           href="/blog/2020/09/05/vue-element-admin/"
           data-tag="web"
           data-author="沐雨云楼" >
            <span class="post-title" title="vue element admin开发">vue element admin开发</span>
            <span class="post-date" title="2020-09-05 11:01:38">2020/09/05</span>
        </a>
        
        <a  class="工具 "
           href="/blog/2020/09/05/tool-notebook/"
           data-tag="notebook"
           data-author="沐雨云楼" >
            <span class="post-title" title="常见笔记软件">常见笔记软件</span>
            <span class="post-date" title="2020-09-05 10:54:34">2020/09/05</span>
        </a>
        
        <a  class="java dao "
           href="/blog/2020/08/30/java-dao-shardingSphere/"
           data-tag="sharding-jdbc"
           data-author="沐雨云楼" >
            <span class="post-title" title="分库分表中间件-ShardingSphere">分库分表中间件-ShardingSphere</span>
            <span class="post-date" title="2020-08-30 21:52:45">2020/08/30</span>
        </a>
        
        <a  class="java dao "
           href="/blog/2020/08/30/java-dao-mycat/"
           data-tag="mycat"
           data-author="沐雨云楼" >
            <span class="post-title" title="分库分表中间件-MyCat">分库分表中间件-MyCat</span>
            <span class="post-date" title="2020-08-30 15:01:59">2020/08/30</span>
        </a>
        
        <a  class="java dao "
           href="/blog/2020/08/30/java-dao-sharding-db/"
           data-tag="分库分表"
           data-author="沐雨云楼" >
            <span class="post-title" title="分库分表中间件-开源产品">分库分表中间件-开源产品</span>
            <span class="post-date" title="2020-08-30 09:37:18">2020/08/30</span>
        </a>
        
        <a  class="java dao "
           href="/blog/2020/08/29/java-jndi/"
           data-tag="jdni"
           data-author="沐雨云楼" >
            <span class="post-title" title="JNDI">JNDI</span>
            <span class="post-date" title="2020-08-29 17:12:03">2020/08/29</span>
        </a>
        
        <a  class="工具 git "
           href="/blog/2020/08/27/tool-git-setting/"
           data-tag="git"
           data-author="沐雨云楼" >
            <span class="post-title" title="git配置">git配置</span>
            <span class="post-date" title="2020-08-27 21:30:17">2020/08/27</span>
        </a>
        
        <a  class="java test "
           href="/blog/2020/08/23/java-powermock/"
           data-tag="powermock"
           data-author="沐雨云楼" >
            <span class="post-title" title="powermock测试框架">powermock测试框架</span>
            <span class="post-date" title="2020-08-23 12:45:17">2020/08/23</span>
        </a>
        
        <a  class="java test "
           href="/blog/2020/08/23/java-mockito/"
           data-tag="mock"
           data-author="沐雨云楼" >
            <span class="post-title" title="👍mockito测试框架">👍mockito测试框架</span>
            <span class="post-date" title="2020-08-23 10:45:17">2020/08/23</span>
        </a>
        
        <a  class="java test "
           href="/blog/2020/08/22/java-test-ng/"
           data-tag="test"
           data-author="沐雨云楼" >
            <span class="post-title" title="👍TestNG一篇足以">👍TestNG一篇足以</span>
            <span class="post-date" title="2020-08-22 18:37:28">2020/08/22</span>
        </a>
        
        <a  class="db mysql "
           href="/blog/2020/08/19/mysql-common-sql/"
           data-tag="mysql"
           data-author="沐雨云楼" >
            <span class="post-title" title="mysql常用sql">mysql常用sql</span>
            <span class="post-date" title="2020-08-19 21:30:00">2020/08/19</span>
        </a>
        
        <a  class="db mysql "
           href="/blog/2020/08/19/mysql-windows-zip-install/"
           data-tag="mysql"
           data-author="沐雨云楼" >
            <span class="post-title" title="mysql zip在windows上安装">mysql zip在windows上安装</span>
            <span class="post-date" title="2020-08-19 21:00:47">2020/08/19</span>
        </a>
        
        <a  class="工具 idea "
           href="/blog/2020/08/15/jetbrains-crack/"
           data-tag="工具,idea"
           data-author="沐雨云楼" >
            <span class="post-title" title="jetbrains全家桶破解">jetbrains全家桶破解</span>
            <span class="post-date" title="2020-08-15 19:22:33">2020/08/15</span>
        </a>
        
        <a  class="工具 git "
           href="/blog/2020/07/10/tool-git-error-name-too-long/"
           data-tag="git"
           data-author="沐雨云楼" >
            <span class="post-title" title="git文件名太长">git文件名太长</span>
            <span class="post-date" title="2020-07-10 12:03:02">2020/07/10</span>
        </a>
        
        <a  class="db mysql "
           href="/blog/2020/07/04/mysql-lock-unlock-table/"
           data-tag="lock"
           data-author="沐雨云楼" >
            <span class="post-title" title="mysql数据库锁查询和释放">mysql数据库锁查询和释放</span>
            <span class="post-date" title="2020-07-04 21:46:54">2020/07/04</span>
        </a>
        
        <a  class="结构和算法 "
           href="/blog/2020/07/04/data-structures-algorithms-sort/"
           data-tag="结构和算法"
           data-author="沐雨云楼" >
            <span class="post-title" title="数据结构和算法8-排序">数据结构和算法8-排序</span>
            <span class="post-date" title="2020-07-04 18:49:12">2020/07/04</span>
        </a>
        
        <a  class="结构和算法 "
           href="/blog/2020/07/04/data-structures-algorithms-search/"
           data-tag="结构和算法"
           data-author="沐雨云楼" >
            <span class="post-title" title="数据结构和算法7-搜索">数据结构和算法7-搜索</span>
            <span class="post-date" title="2020-07-04 17:48:19">2020/07/04</span>
        </a>
        
        <a  class="结构和算法 "
           href="/blog/2020/07/04/data-structures-algorithms-graph/"
           data-tag="结构和算法"
           data-author="沐雨云楼" >
            <span class="post-title" title="数据结构和算法6-非线性-图">数据结构和算法6-非线性-图</span>
            <span class="post-date" title="2020-07-04 16:09:47">2020/07/04</span>
        </a>
        
        <a  class="结构和算法 "
           href="/blog/2020/07/04/data-structures-algorithms-tree/"
           data-tag="结构和算法"
           data-author="沐雨云楼" >
            <span class="post-title" title="数据结构和算法5-非线性-树">数据结构和算法5-非线性-树</span>
            <span class="post-date" title="2020-07-04 15:09:47">2020/07/04</span>
        </a>
        
        <a  class="结构和算法 "
           href="/blog/2020/07/04/data-structures-algorithms-linear-queue/"
           data-tag="结构和算法"
           data-author="沐雨云楼" >
            <span class="post-title" title="数据结构和算法4-线性-队列">数据结构和算法4-线性-队列</span>
            <span class="post-date" title="2020-07-04 14:10:22">2020/07/04</span>
        </a>
        
        <a  class="结构和算法 "
           href="/blog/2020/07/04/data-structures-algorithms-linear-stack/"
           data-tag="结构和算法"
           data-author="沐雨云楼" >
            <span class="post-title" title="数据结构和算法3-线性-栈">数据结构和算法3-线性-栈</span>
            <span class="post-date" title="2020-07-04 13:00:17">2020/07/04</span>
        </a>
        
        <a  class="结构和算法 "
           href="/blog/2020/07/04/data-structures-algorithms-linear-list/"
           data-tag="结构和算法"
           data-author="沐雨云楼" >
            <span class="post-title" title="数据结构和算法2-线性-链">数据结构和算法2-线性-链</span>
            <span class="post-date" title="2020-07-04 11:32:12">2020/07/04</span>
        </a>
        
        <a  class="结构和算法 "
           href="/blog/2020/07/04/data-structures-algorithms-info/"
           data-tag="结构和算法"
           data-author="沐雨云楼" >
            <span class="post-title" title="数据结构和算法1-介绍">数据结构和算法1-介绍</span>
            <span class="post-date" title="2020-07-04 10:10:40">2020/07/04</span>
        </a>
        
        <a  class="大数据 spark "
           href="/blog/2020/06/30/spark-install-windows/"
           data-tag="spark"
           data-author="沐雨云楼" >
            <span class="post-title" title="spark开发环境搭建">spark开发环境搭建</span>
            <span class="post-date" title="2020-06-30 19:24:51">2020/06/30</span>
        </a>
        
        <a  class="大数据 hadoop "
           href="/blog/2020/06/30/hadoop-install-windows/"
           data-tag="hadoop"
           data-author="沐雨云楼" >
            <span class="post-title" title="hadoop开发环境搭建">hadoop开发环境搭建</span>
            <span class="post-date" title="2020-06-30 18:06:23">2020/06/30</span>
        </a>
        
        <a  class="java "
           href="/blog/2020/06/30/java-install-windows/"
           data-tag="java"
           data-author="沐雨云楼" >
            <span class="post-title" title="jdk开发环境搭建">jdk开发环境搭建</span>
            <span class="post-date" title="2020-06-30 15:09:34">2020/06/30</span>
        </a>
        
        <a  class="ai "
           href="/blog/2020/06/30/ai-python-spark-env/"
           data-tag="spark"
           data-author="沐雨云楼" >
            <span class="post-title" title="python spark开发环境搭建">python spark开发环境搭建</span>
            <span class="post-date" title="2020-06-30 12:14:04">2020/06/30</span>
        </a>
        
        <a  class="ai "
           href="/blog/2020/06/29/ai-jupyter-notebook/"
           data-tag="标签"
           data-author="沐雨云楼" >
            <span class="post-title" title="jupyter notebook">jupyter notebook</span>
            <span class="post-date" title="2020-06-29 12:29:36">2020/06/29</span>
        </a>
        
        <a  class="ai "
           href="/blog/2020/06/26/ai-python-tools/"
           data-tag="tools"
           data-author="沐雨云楼" >
            <span class="post-title" title="python和ai常用工具">python和ai常用工具</span>
            <span class="post-date" title="2020-06-26 09:55:24">2020/06/26</span>
        </a>
        
        <a  class="python "
           href="/blog/2020/06/25/python-tool-buildin-attr/"
           data-tag="attr"
           data-author="沐雨云楼" >
            <span class="post-title" title="👍python内置属性(魔法方法)">👍python内置属性(魔法方法)</span>
            <span class="post-date" title="2020-06-25 11:01:33">2020/06/25</span>
        </a>
        
        <a  class="python "
           href="/blog/2020/06/24/python-tool-buildin-func/"
           data-tag="func"
           data-author="沐雨云楼" >
            <span class="post-title" title="👍python内置函数">👍python内置函数</span>
            <span class="post-date" title="2020-06-24 18:31:59">2020/06/24</span>
        </a>
        
        <a  class="python "
           href="/blog/2020/06/24/python-tool-obj-dict/"
           data-tag="dict"
           data-author="沐雨云楼" >
            <span class="post-title" title="python对象与dict互转">python对象与dict互转</span>
            <span class="post-date" title="2020-06-24 18:15:03">2020/06/24</span>
        </a>
        
        <a  class="大数据 spark "
           href="/blog/2020/06/21/spark-version-3/"
           data-tag="spark"
           data-author="沐雨云楼" >
            <span class="post-title" title="spark3.0新特性">spark3.0新特性</span>
            <span class="post-date" title="2020-06-21 17:40:12">2020/06/21</span>
        </a>
        
        <a  class="java "
           href="/blog/2020/06/21/java-base-domain/"
           data-tag="domain"
           data-author="沐雨云楼" >
            <span class="post-title" title="VO DTO DO PO你都了解吗?">VO DTO DO PO你都了解吗?</span>
            <span class="post-date" title="2020-06-21 09:20:03">2020/06/21</span>
        </a>
        
        <a  class="大数据 spark "
           href="/blog/2020/06/20/spark-data-skew/"
           data-tag="spark"
           data-author="沐雨云楼" >
            <span class="post-title" title="spark数据倾斜及解决方案">spark数据倾斜及解决方案</span>
            <span class="post-date" title="2020-06-20 21:41:12">2020/06/20</span>
        </a>
        
        <a  class="linux nginx "
           href="/blog/2020/06/20/nginx-fast/"
           data-tag="nginx"
           data-author="沐雨云楼" >
            <span class="post-title" title="nginx为何这么快?">nginx为何这么快?</span>
            <span class="post-date" title="2020-06-20 17:55:10">2020/06/20</span>
        </a>
        
        <a  class="架构 "
           href="/blog/2020/06/20/fw-mq-message-missing/"
           data-tag="mq"
           data-author="沐雨云楼" >
            <span class="post-title" title="MQ如何保证消息不丢失?">MQ如何保证消息不丢失?</span>
            <span class="post-date" title="2020-06-20 17:03:18">2020/06/20</span>
        </a>
        
        <a  class="java "
           href="/blog/2020/06/20/java-base-serializable/"
           data-tag="序列化"
           data-author="沐雨云楼" >
            <span class="post-title" title="java序列化">java序列化</span>
            <span class="post-date" title="2020-06-20 16:29:04">2020/06/20</span>
        </a>
        
        <a  class="架构 "
           href="/blog/2020/06/20/fw-distributed-lock-redis-zk/"
           data-tag="lock,redis,zookeeper"
           data-author="沐雨云楼" >
            <span class="post-title" title="分布式锁用redis还是zk?">分布式锁用redis还是zk?</span>
            <span class="post-date" title="2020-06-20 12:09:55">2020/06/20</span>
        </a>
        
        <a  class="java tools "
           href="/blog/2020/06/19/java-tool-jackson/"
           data-tag="jackson"
           data-author="沐雨云楼" >
            <span class="post-title" title="工具类--jackson">工具类--jackson</span>
            <span class="post-date" title="2020-06-19 13:00:45">2020/06/19</span>
        </a>
        
        <a  class="java tools "
           href="/blog/2020/06/19/java-tool-bean-map/"
           data-tag="beanmap"
           data-author="沐雨云楼" >
            <span class="post-title" title="工具类--bean和map互转">工具类--bean和map互转</span>
            <span class="post-date" title="2020-06-19 12:30:14">2020/06/19</span>
        </a>
        
        <a  class="java test "
           href="/blog/2020/06/18/java-rest-assured-in-action/"
           data-tag="rest"
           data-author="沐雨云楼" >
            <span class="post-title" title="rest-assured实战">rest-assured实战</span>
            <span class="post-date" title="2020-06-18 17:51:56">2020/06/18</span>
        </a>
        
        <a  class="java test "
           href="/blog/2020/06/17/java_rest_assured_wiki_info/"
           data-tag="rest"
           data-author="沐雨云楼" >
            <span class="post-title" title="rest-assured wiki翻译">rest-assured wiki翻译</span>
            <span class="post-date" title="2020-06-17 20:39:11">2020/06/17</span>
        </a>
        
        <a  class="python tornado "
           href="/blog/2020/06/16/tornado-wtforms/"
           data-tag="wtforms"
           data-author="沐雨云楼" >
            <span class="post-title" title="wtforms表单验证">wtforms表单验证</span>
            <span class="post-date" title="2020-06-16 21:52:33">2020/06/16</span>
        </a>
        
        <a  class="python tornado "
           href="/blog/2020/06/16/tornado-sync-to-async/"
           data-tag="aysnc"
           data-author="沐雨云楼" >
            <span class="post-title" title="tornado同步转异步">tornado同步转异步</span>
            <span class="post-date" title="2020-06-16 21:50:25">2020/06/16</span>
        </a>
        
        <a  class="python tornado "
           href="/blog/2020/06/08/python_tornado_info/"
           data-tag="tornado"
           data-author="沐雨云楼" >
            <span class="post-title" title="👍Tornado入门这一篇足以">👍Tornado入门这一篇足以</span>
            <span class="post-date" title="2020-06-08 20:31:32">2020/06/08</span>
        </a>
        
        <a  class="工具 maven "
           href="/blog/2020/06/07/tool-maven/"
           data-tag="tools,工具"
           data-author="沐雨云楼" >
            <span class="post-title" title="maven常用命令和配置">maven常用命令和配置</span>
            <span class="post-date" title="2020-06-07 16:30:11">2020/06/07</span>
        </a>
        
        <a  class="微服务 springboot "
           href="/blog/2020/06/07/spring-initialzr/"
           data-tag="tools,工具"
           data-author="沐雨云楼" >
            <span class="post-title" title="spring initialzr环境编译发布">spring initialzr环境编译发布</span>
            <span class="post-date" title="2020-06-07 15:06:00">2020/06/07</span>
        </a>
        
        <a  class="windows "
           href="/blog/2020/06/07/windows-cmd/"
           data-tag="tools,工具,命令"
           data-author="沐雨云楼" >
            <span class="post-title" title="windows常用命令和技巧">windows常用命令和技巧</span>
            <span class="post-date" title="2020-06-07 11:39:01">2020/06/07</span>
        </a>
        
        <a  class="工具 github "
           href="/blog/2020/06/07/tool-connect-github/"
           data-tag="github"
           data-author="沐雨云楼" >
            <span class="post-title" title="github无法访问解决办法">github无法访问解决办法</span>
            <span class="post-date" title="2020-06-07 10:08:38">2020/06/07</span>
        </a>
        
        <a  class="微服务 spring "
           href="/blog/2020/06/03/spring-transaction/"
           data-tag="transactional"
           data-author="沐雨云楼" >
            <span class="post-title" title="Transactional注解">Transactional注解</span>
            <span class="post-date" title="2020-06-03 22:09:42">2020/06/03</span>
        </a>
        
        <a  class="java dao "
           href="/blog/2020/06/03/java-jpa-query-multi-param/"
           data-tag="JPA"
           data-author="沐雨云楼" >
            <span class="post-title" title="JPA多条件查询">JPA多条件查询</span>
            <span class="post-date" title="2020-06-03 21:38:04">2020/06/03</span>
        </a>
        
        <a  class="java tools "
           href="/blog/2020/06/02/java-tool-fastjson-bug/"
           data-tag="fastjson"
           data-author="沐雨云楼" >
            <span class="post-title" title="Fastjson远程代码高危漏洞">Fastjson远程代码高危漏洞</span>
            <span class="post-date" title="2020-06-02 22:38:28">2020/06/02</span>
        </a>
        
        <a  class="linux docker "
           href="/blog/2020/06/02/docker-introduction/"
           data-tag="docker"
           data-author="沐雨云楼" >
            <span class="post-title" title="👍docker入门看这一篇就够了">👍docker入门看这一篇就够了</span>
            <span class="post-date" title="2020-06-02 22:22:30">2020/06/02</span>
        </a>
        
        <a  class="面试 "
           href="/blog/2020/06/02/interview-java-question/"
           data-tag="面试"
           data-author="沐雨云楼" >
            <span class="post-title" title="java常见面试题">java常见面试题</span>
            <span class="post-date" title="2020-06-02 22:15:48">2020/06/02</span>
        </a>
        
        <a  class="微服务 "
           href="/blog/2020/06/02/microservice-why/"
           data-tag="微服务"
           data-author="沐雨云楼" >
            <span class="post-title" title="牛逼公司一定要使用微服务？">牛逼公司一定要使用微服务？</span>
            <span class="post-date" title="2020-06-02 21:15:29">2020/06/02</span>
        </a>
        
        <a  class="面试 "
           href="/blog/2020/06/02/interview-company/"
           data-tag="面试"
           data-author="沐雨云楼" >
            <span class="post-title" title="IT互联网公司名单">IT互联网公司名单</span>
            <span class="post-date" title="2020-06-02 20:41:23">2020/06/02</span>
        </a>
        
        <a  class="java 并发 "
           href="/blog/2020/06/02/java-concurrent-aqs/"
           data-tag="java,并发"
           data-author="沐雨云楼" >
            <span class="post-title" title="java并发AQS">java并发AQS</span>
            <span class="post-date" title="2020-06-02 20:01:48">2020/06/02</span>
        </a>
        
        <a  class="ai "
           href="/blog/2020/05/30/ai-learn-catalog/"
           data-tag="ai"
           data-author="沐雨云楼" >
            <span class="post-title" title="AI学习目录">AI学习目录</span>
            <span class="post-date" title="2020-05-30 08:49:39">2020/05/30</span>
        </a>
        
        <a  class="工具 hexo "
           href="/blog/2020/05/29/hexo-theme-math/"
           data-tag="工具,hexo"
           data-author="沐雨云楼" >
            <span class="post-title" title="hexo主题支持MathJax">hexo主题支持MathJax</span>
            <span class="post-date" title="2020-05-29 09:36:46">2020/05/29</span>
        </a>
        
        <a  class="db redis "
           href="/blog/2020/05/28/redis-memory-eliminate/"
           data-tag="redis"
           data-author="沐雨云楼" >
            <span class="post-title" title="Redis的内存淘汰策略">Redis的内存淘汰策略</span>
            <span class="post-date" title="2020-05-28 21:37:04">2020/05/28</span>
        </a>
        
        <a  class="工具 markdowm "
           href="/blog/2020/05/28/markdown-math-express/"
           data-tag="工具,markdowm"
           data-author="沐雨云楼" >
            <span class="post-title" title="markdown数学公式">markdown数学公式</span>
            <span class="post-date" title="2020-05-28 08:02:03">2020/05/28</span>
        </a>
        
        <a  class="python "
           href="/blog/2020/05/21/python_install/"
           data-tag="python"
           data-author="沐雨云楼" >
            <span class="post-title" title="python环境安装">python环境安装</span>
            <span class="post-date" title="2020-05-21 21:37:04">2020/05/21</span>
        </a>
        
        <a  class="python "
           href="/blog/2020/05/21/python_base/"
           data-tag="python"
           data-author="沐雨云楼" >
            <span class="post-title" title="python基本语法">python基本语法</span>
            <span class="post-date" title="2020-05-21 00:00:00">2020/05/21</span>
        </a>
        
        <a  class="其他 "
           href="/blog/2020/05/20/todo-list/"
           data-tag="plan,计划"
           data-author="沐雨云楼" >
            <span class="post-title" title="计划">计划</span>
            <span class="post-date" title="2020-05-20 11:04:19">2020/05/20</span>
        </a>
        
        <a  class="工具 "
           href="/blog/2020/05/20/blog-template/"
           data-tag="模板"
           data-author="沐雨云楼" >
            <span class="post-title" title="博客自定义模板">博客自定义模板</span>
            <span class="post-date" title="2020-05-20 05:20:00">2020/05/20</span>
        </a>
        
    </nav>
</div>

    </div>
    <div class="hide-list">
        <div class="semicircle" data-title="切换全屏 快捷键 s">
            <div class="brackets first"><</div>
            <div class="brackets">&gt;</div>
        </div>
    </div>
</aside>
<div class="post">
    <div class="pjax">
        <article id="post-spark-version-3" class="article article-type-post" itemscope itemprop="blogPost">
    
        <h1 class="article-title">spark3.0新特性</h1>
    
    <div class="article-meta">
        
        
        <span class="author"><a>沐雨云楼</a></span>
        
        
        <span class="book">
            
                <a  data-rel="大数据">大数据</a>/
            
                <a  data-rel="spark">spark</a>
            
        </span>
        
        
        <span class="tag">
            
            <a class="color1">spark</a>
            
        </span>
        
    </div>
    <div class="article-meta">
        
            发布时间 : <time class="date" title='最后更新: 2020-09-12 21:21:47'>2020-06-21 17:40</time>
        
    </div>
    <div class="article-meta">
        
        <span>字数:4.3k</span>
        
        
        <span id="busuanzi_container_page_pv">
            阅读 :<span id="busuanzi_value_page_pv">
                <span class="count-comment">
                    <span class="spinner">
                      <div class="cube1"></div>
                      <div class="cube2"></div>
                    </span>
                </span>
            </span>
        </span>
        
        
        <span class="top-comment" title="跳转至评论区">
            <a href="#comments">
                评论:<span class="count-comment">
                    <span class="spinner">
                      <div class="cube1"></div>
                      <div class="cube2"></div>
                    </span>
                </span>
            </a>
        </span>
        
    </div>
    
    <div class="article-meta">
        <span class="origin">来源:
            <a href="https://www.cnblogs.com/w397090770/p/11875829.html" target="_blank">
                
                Apache Spark 3.0 预览版正式发布
                
            </a>
        </span>
    </div>
    
    <div class="toc-ref">
    
        <ol class="toc"><li class="toc-item toc-level-2"><a class="toc-link" href="#1-简介"><span class="toc-text">1.简介</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#2-动态分区修剪"><span class="toc-text">2.动态分区修剪</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#2-自适应查询执行"><span class="toc-text">2.自适应查询执行</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#3-加速器感知调度"><span class="toc-text">3.加速器感知调度</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#4-DataSource-V2"><span class="toc-text">4.DataSource V2</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#5-丰富的API和功能"><span class="toc-text">5.丰富的API和功能</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#5-1-增强的pandas-UDF"><span class="toc-text">5-1.增强的pandas UDF</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#5-2-一组完整的-join-hints"><span class="toc-text">5-2.一组完整的 join hints</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#5-3-新的内置函数"><span class="toc-text">5-3.新的内置函数</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#6-增强监控功能"><span class="toc-text">6.增强监控功能</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#6-1-重新设计Structured-streaming的-UI"><span class="toc-text">6-1.重新设计Structured streaming的 UI</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#6-2-增强EXPLAIN命令"><span class="toc-text">6-2.增强EXPLAIN命令</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#6-3-可观察的指标"><span class="toc-text">6-3.可观察的指标</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#7-更好的ANSI-SQL兼容"><span class="toc-text">7.更好的ANSI SQL兼容</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#8-SparkR向量化读写"><span class="toc-text">8.SparkR向量化读写</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#9-Kafka-Streaming-includeHeaders"><span class="toc-text">9.Kafka Streaming: includeHeaders</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#10-其他"><span class="toc-text">10.其他</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#11-参考"><span class="toc-text">11.参考</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#12-公众号"><span class="toc-text">12.公众号</span></a></li></ol>
    
<style>
    .left-col .switch-btn,
    .left-col .switch-area {
        display: none;
    }
    .toc-level-4 i,
    .toc-level-4 ol {
        display: none !important;
    }
</style>
</div>
    
    <div class="article-entry" itemprop="articleBody">
      
        <p><strong>主要特性</strong></p>
<ul>
<li>包括动态分区修剪（Dynamic Partition Pruning）</li>
<li>自适应查询执行（Adaptive Query Execution）</li>
<li>加速器感知调度（Accelerator-aware Scheduling）</li>
<li>支持 Catalog 的数据源API（Data Source API with Catalog Supports）</li>
<li>SparkR中的向量化（Vectorization in SparkR）</li>
</ul>
<h2 id="1-简介"><a href="#1-简介" class="headerlink" title="1.简介"></a>1.简介</h2><p>原计划在2019年年底发布的 Apache Spark 3.0.0 今天终于赶在下周二举办的 Spark Summit AI 会议之前正式发布了! Apache Spark 3.0.0 自2018年10月02日开发到目前已经经历了近21个月！这个版本的发布经历了两个预览版以及三次投票：</p>
<ul>
<li>2019年11月06日第一次预览版，参见Preview release of Spark 3.0[1]</li>
<li>2019年12月23日第二次预览版，参见Preview release of Spark 3.0[2]</li>
<li>2020年03月21日 [VOTE] Apache Spark 3.0.0 RC1[3]</li>
<li>2020年05月18日 [VOTE] Apache Spark 3.0 RC2[4]</li>
<li>2020年06月06日 [vote] Apache Spark 3.0 RC3[5]</li>
</ul>
<p>Apache Spark 3.0 增加了很多令人兴奋的新特性，包括动态分区修剪（Dynamic Partition Pruning）、自适应查询执行（Adaptive Query Execution）、加速器感知调度（Accelerator-aware Scheduling）、支持 Catalog 的数据源API（Data Source API with Catalog Supports）、SparkR 中的向量化（Vectorization in SparkR）、支持 Hadoop 3/JDK 11/Scala 2.12 等等。这个版本一共解决了 3400 多个 ISSUES。</p>
<p><strong>issues</strong></p>
<p><img src="/blog/images/bigdata/spark/spark_3_issues.jpg" alt="img"></p>
<div class="admonition note">
<p class="fa admonition-title">没多少Streaming/Structed Streaming 相关的ISSUE</p>
<p>
    <li>
        目前基于 Batch 模式的 Spark Streaming/Structed Streaming 能够满足企业大部分的需求，真正需要非常实时计算的应用还是很少的，所以 Continuous Processing 模块还处于试验阶段，还不急着毕业；
    </li>
    <li>
        数砖应该在大量投入开发 Delta Lake 相关的东西，这个能够企业带来收入，目前这个才是他们的重点，所以自然开发 Streaming 的投入少了。 好了，废话不扯了，我们来看看 Spark 3.0 的新特性吧。
    </li>
</p>
</div>


<p><strong>特新</strong></p>
<img src="/blog/images/bigdata/spark/spark_3_feature.jpg" width="800px">

<h2 id="2-动态分区修剪"><a href="#2-动态分区修剪" class="headerlink" title="2.动态分区修剪"></a>2.动态分区修剪</h2><p>Dynamic Partition Pruning</p>
<p>所谓的动态分区裁剪就是基于运行时（run time）推断出来的信息来进一步进行分区裁剪。举个例子，我们有如下的查询：</p>
<pre><code class="sql">SELECT * FROM dim_iteblog 
JOIN fact_iteblog 
ON (dim_iteblog.partcol = fact_iteblog.partcol) 
WHERE dim_iteblog.othercol &gt; 10</code></pre>
<p>假设dim_iteblog表<code>dim_iteblog.othercol &gt; 10</code>过滤出来的数据比较少，但是由于之前版本的 Spark 无法进行动态计算代价，所以可能会导致 fact_iteblog 表扫描出大量无效的数据。有了动态分区裁减，可以在运行的时候过滤掉 fact_iteblog 表无用的数据。经过这个优化，查询扫描的数据大大减少，性能提升了 33 倍。</p>
<p>在TPC-DS基准测试中，102个查询中的60个得到2到18倍的加速。</p>
<img src="/blog/images/bigdata/spark/spark_3_dynamic_partition_pruning_tcpds.png" width="650px">


<p>这个特性对应的ISSUE可以参见 SPARK-11150 和 SPARK-28888。过往记忆大数据公众号也在前段时间对这个特性进行了详细介绍，具体请参见《Apache Spark 3.0 动态分区裁剪（Dynamic Partition Pruning）介绍》和《Apache Spark 3.0 动态分区裁剪（Dynamic Partition Pruning）使用》。</p>
<h2 id="2-自适应查询执行"><a href="#2-自适应查询执行" class="headerlink" title="2.自适应查询执行"></a>2.自适应查询执行</h2><p>Adaptive Query Execution</p>
<p>自适应查询执行（又称 Adaptive Query Optimisation 或者 Adaptive Optimisation）是对查询执行计划的优化，允许 Spark Planner 在运行时执行可选的执行计划，这些计划将基于运行时统计数据进行优化。</p>
<p>早在2015年，Spark 社区就提出了自适应执行的基本想法，在 Spark 的 DAGScheduler 中增加了提交单个 map stage 的接口，并且在实现运行时调整 shuffle partition 数量上做了尝试。但目前该实现有一定的局限性，在某些场景下会引入更多的 shuffle，即更多的 stage，对于三表在同一个 stage 中做 join 等情况也无法很好的处理；而且使用当前框架很难灵活地在自适应执行中实现其他功能，例如更改执行计划或在运行时处理倾斜的 join。所以该功能一直处于实验阶段，配置参数也没有在官方文档中提及。这个想法主要来自英特尔以及百度的大牛，具体参见 SPARK-9850，对应的文章可以参见《Apache Spark SQL自适应执行实践》。</p>
<p>而 Apache Spark 3.0 的 Adaptive Query Execution（AQE） 是基于 SPARK-9850 的思想而实现的，具体参见 SPARK-23128。SPARK-23128 的目标是实现一个灵活的框架以在 Spark SQL 中执行自适应执行，并支持在运行时更改 reducer 的数量。新的实现解决了前面讨论的所有限制，其他功能（如更改 join 策略和处理倾斜 join）将作为单独的功能实现，并作为插件在后面版本提供。</p>
<p>AQE 框架目前提供了三个功能：</p>
<ul>
<li>动态合并 shuffle partitions；</li>
<li>动态调整 join 策略；</li>
<li>动态优化倾斜的 join（skew joins）。</li>
</ul>
<p>基于没有统计数据的 1TB TPC-DS 基准，Spark 3.0 可以使 q77 的速度提高8倍，使 q5 的速度提高2倍，而对另外26个查询的速度提高1.1倍以上。可以通过设置 SQL 配置 spark.sql.adaptive=true 来启用 AQE，这个参数默认值为 false。</p>
<img src="/blog/images/bigdata/spark/spark_3_adaptive_query_execution.png" width="650px">


<h2 id="3-加速器感知调度"><a href="#3-加速器感知调度" class="headerlink" title="3.加速器感知调度"></a>3.加速器感知调度</h2><p>Accelerator-aware Scheduling</p>
<p>如今大数据和机器学习已经有了很大的结合，在机器学习里面，因为计算迭代的时间可能会很长，开发人员一般会选择使用 GPU、FPGA 或 TPU 来加速计算。在 Apache Hadoop 3.1 版本里面已经开始内置原生支持 GPU 和 FPGA 了。作为通用计算引擎的 Spark 肯定也不甘落后，来自 Databricks、NVIDIA、Google 以及阿里巴巴的工程师们正在为 Apache Spark 添加原生的 GPU 调度支持，该方案填补了 Spark 在 GPU 资源的任务调度方面的空白，有机地融合了大数据处理和 AI 应用，扩展了 Spark 在深度学习、信号处理和各大数据应用的应用场景。这项工作的 issue 可以在 SPARK-24615 里面查看，相关的 SPIP（Spark Project Improvement Proposals） 文档可以参见 SPIP: Accelerator-aware scheduling[6]</p>
<p><strong>Apache Spark 3.0 将内置支持GPU调度</strong></p>
<img src="/blog/images/bigdata/spark/spark_3_accelerator_aware_scheduling.png" width="450px">



<p>目前 Apache Spark 支持的资源管理器 YARN 和 Kubernetes 已经支持了 GPU。为了让 Spark 也支持 GPUs，在技术层面上需要做出两个主要改变：</p>
<ul>
<li>在 cluster manager 层面上，需要升级 cluster managers 来支持 GPU。并且给用户提供相关 API，使得用户可以控制 GPU 资源的使用和分配。</li>
<li>在 Spark 内部，需要在 scheduler 层面做出修改，使得 scheduler 可以在用户 task 请求中识别 GPU 的需求，然后根据 executor 上的 GPU 供给来完成分配。</li>
</ul>
<p>因为让 Apache Spark 支持 GPU 是一个比较大的特性，所以项目分为了几个阶段。在 Apache Spark 3.0 版本，将支持在 standalone、 YARN 以及 Kubernetes 资源管理器下支持 GPU，并且对现有正常的作业基本没影响。对于 GPU 的支持、Mesos 资源管理器中 GPU 的支持、以及 Windows 平台的 GPU 支持将不是这个版本的目标。而且对于一张 GPU 卡内的细粒度调度也不会在这个版本支持；Apache Spark 3.0 版本将把一张 GPU 卡和其内存作为不可分割的单元。详情请参见 过往记忆大数据公众号的《Apache Spark 3.0 将内置支持 GPU 调度》文章。</p>
<h2 id="4-DataSource-V2"><a href="#4-DataSource-V2" class="headerlink" title="4.DataSource V2"></a>4.DataSource V2</h2><p>Data Source API 定义如何从存储系统进行读写的相关 API 接口，比如 Hadoop 的 InputFormat/OutputFormat，Hive 的 Serde 等。这些 API 非常适合用户在 Spark 中使用 RDD 编程的时候使用。使用这些 API 进行编程虽然能够解决我们的问题，但是对用户来说使用成本还是挺高的，而且 Spark 也不能对其进行优化。为了解决这些问题，Spark 1.3 版本开始引入了 Data Source API V1，通过这个 API 我们可以很方便的读取各种来源的数据，而且 Spark 使用 SQL 组件的一些优化引擎对数据源的读取进行优化，比如列裁剪、过滤下推等等。</p>
<img src="/blog/images/bigdata/spark/spark_3_datasource_v2.png" width="800px">

<p>Data Source API V1 为我们抽象了一系列的接口，使用这些接口可以实现大部分的场景。但是随着使用的用户增多，逐渐显现出一些问题：</p>
<ul>
<li>部分接口依赖 SQLContext 和 DataFrame</li>
<li>扩展能力有限，难以下推其他算子</li>
<li>缺乏对列式存储读取的支持</li>
<li>缺乏分区和排序信息</li>
<li>写操作不支持事务</li>
<li>不支持流处理</li>
</ul>
<p>为了解决 Data Source V1 的一些问题，从 Apache Spark 2.3.0 版本开始，社区引入了 Data Source API V2，在保留原有的功能之外，还解决了 Data Source API V1 存在的一些问题，比如不再依赖上层 API，扩展能力增强。Data Source API V2 对应的 ISSUE 可以参见 SPARK-15689。虽然这个功能在 Apache Spark 2.x 版本就出现了，但是不是很稳定，所以社区对 Spark DataSource API V2 的稳定性工作以及新功能分别开了两个 ISSUE：SPARK-25186 以及 SPARK-22386。Spark DataSource API V2 最终稳定版以及新功能将会随着年底和 Apache Spark 3.0.0 版本一起发布，其也算是 Apache Spark 3.0.0 版本的一大新功能。</p>
<p>更多关于 Apache Spark DataSource V2 的详细介绍请参见<strong>过往记忆大数据</strong>公众号的 Apache Spark DataSource V2 介绍及入门编程指南（上） 和 Apache Spark DataSource V2 介绍及入门编程指南（下） 两篇文章的介绍。</p>
<h2 id="5-丰富的API和功能"><a href="#5-丰富的API和功能" class="headerlink" title="5.丰富的API和功能"></a>5.丰富的API和功能</h2><p>为了满足新的用例并简化 Spark 应用程序的开发，Apache Spark 3.0 版本提供了新的功能并增强了现有的功能。</p>
<h3 id="5-1-增强的pandas-UDF"><a href="#5-1-增强的pandas-UDF" class="headerlink" title="5-1.增强的pandas UDF"></a>5-1.增强的pandas UDF</h3><p>Pandas UDF 最初是在 Spark 2.3 中引入的，用于扩展 PySpark 中 UDF 并将 pandas API 集成到 PySpark 应用程序中。但是，当添加更多 UDF 类型时，现有接口很难理解。为了解决这个问题，Spark 3.0 引入了带有 Python 类型提示的新 pandas UDF 接口。此版本增加了两种新的 pandas UDF 类型：iterator of series to iterator of series 和 iterator of multiple series to iterator of series，以及三个新的 pandas 函数 API：grouped map、map 和 co-grouped map。详细介绍可以参见过往记忆大数据的 Apache Spark 3.0 新的 Pandas UDF 及 Python Type Hints：<a href="https://www.iteblog.com/archives/9814.html。" target="_blank" rel="noopener">https://www.iteblog.com/archives/9814.html。</a></p>
<h3 id="5-2-一组完整的-join-hints"><a href="#5-2-一组完整的-join-hints" class="headerlink" title="5-2.一组完整的 join hints"></a>5-2.一组完整的 join hints</h3><p>尽管社区不断提高编译器的智能性，但不能保证编译器始终可以针对每种情况做出最佳决策。Join 算法的选择基于统计和启发式算法，当编译器无法做出最佳选择时，用户仍然可以使用 join hints 来影响优化器选择更好的计划。Apache Spark 3.0 通过添加新的 hints 扩展了现有的 join hints ：SHUFFLE_MERGE、SHUFFLE_HASH 和 SHUFFLE_REPLICATE_NL</p>
<h3 id="5-3-新的内置函数"><a href="#5-3-新的内置函数" class="headerlink" title="5-3.新的内置函数"></a>5-3.新的内置函数</h3><p>Scala API 中增了32个新的内置函数和高阶函数。在这些内置函数中，添加了一组针对 MAP 的特定内置函数[transform_key，transform_value，map_entries，map_filter，map_zip_with]，以简化对 MAP 数据类型的处理。</p>
<h2 id="6-增强监控功能"><a href="#6-增强监控功能" class="headerlink" title="6.增强监控功能"></a>6.增强监控功能</h2><p>Apache Spark 在监控方面也包含许多增强功能，这些功能使监控更加全面和稳定。这些增强的监控功能不会对性能产生重大影响。主要可以分为以下三个地方。</p>
<h3 id="6-1-重新设计Structured-streaming的-UI"><a href="#6-1-重新设计Structured-streaming的-UI" class="headerlink" title="6-1.重新设计Structured streaming的 UI"></a>6-1.重新设计Structured streaming的 UI</h3><p>Structured streaming 最初是在 Spark 2.0 中引入的。Spark 3.0 为监控这些流作业重新设计了 UI。这个新的 UI 提供了两组统计信息：</p>
<ul>
<li>已完成的流查询作业的聚合信息</li>
<li>流查询的详细统计信息，包括 Input Rate, Process Rate, Input Rows, Batch Duration, Operation Duration 等</li>
</ul>
<img src="/blog/images/bigdata/spark/spark_3_streaming_ui.png" width="650px">


<h3 id="6-2-增强EXPLAIN命令"><a href="#6-2-增强EXPLAIN命令" class="headerlink" title="6-2.增强EXPLAIN命令"></a>6-2.增强EXPLAIN命令</h3><p>读取计划（Reading plans）对理解和调优查询非常重要。现有的解决方案看起来很混乱，每个算子的字符串表示可能非常宽，甚至可能被截断。Spark 3.0 版本使用一种新的格式化（FORMATTED）模式对其进行了增强，并且还提供了将计划转储到文件的功能。</p>
<h3 id="6-3-可观察的指标"><a href="#6-3-可观察的指标" class="headerlink" title="6-3.可观察的指标"></a>6-3.可观察的指标</h3><p>连续监视数据质量的变化是管理数据管道非常需要的特性。Spark 3.0 版本为批处理和流处理应用程序引入了这种功能。可观察指标被命名为可以在查询上定义的任意聚合函数(dataframe)。一旦 dataframe 的执行到达一个完成点（例如，完成批查询），就会发出一个命名事件，其中包含自上一个完成点以来处理的数据的指标。</p>
<h2 id="7-更好的ANSI-SQL兼容"><a href="#7-更好的ANSI-SQL兼容" class="headerlink" title="7.更好的ANSI SQL兼容"></a>7.更好的ANSI SQL兼容</h2><p>PostgreSQL 是最先进的开源数据库之一，其支持 SQL:2011 的大部分主要特性，完全符合 SQL:2011 要求的 179 个功能中，PostgreSQL 至少符合 160 个。Spark 社区目前专门开了一个 ISSUE SPARK-27764 来解决 Spark SQL 和 PostgreSQL 之间的差异，包括功能特性补齐、Bug 修改等。功能补齐包括了支持 ANSI SQL 的一些函数、区分 SQL 保留关键字以及内置函数等。这个 ISSUE 下面对应了 231 个子 ISSUE，如果这部分的 ISSUE 都解决了，那么 Spark SQL 和 PostgreSQL 或者 ANSI SQL:2011 之间的差异更小了。</p>
<h2 id="8-SparkR向量化读写"><a href="#8-SparkR向量化读写" class="headerlink" title="8.SparkR向量化读写"></a>8.SparkR向量化读写</h2><p>Spark 是从 1.4 版本开始支持 R 语言的，但是那时候 Spark 和 R 进行交互的架构图如下：</p>
<img src="/blog/images/bigdata/spark/spark_3_spark_r_old.png" width="450px">


<p>每当我们使用 R 语言和 Spark 集群进行交互，需要经过 JVM ，这也就无法避免数据的序列化和反序列化操作，这在数据量很大的情况下性能是十分低下的！</p>
<p>而且 Apache Spark 已经在许多操作中进行了向量化优化（vectorization optimization），例如，内部列式格式（columnar format）、Parquet/ORC 向量化读取、Pandas UDFs 等。向量化可以大大提高性能。SparkR 向量化允许用户按原样使用现有代码，但是当他们执行 R 本地函数或将 Spark DataFrame 与 R DataFrame 互相转换时，可以将性能提高大约数千倍。这项工作可以看下 SPARK-26759。新的架构如下:</p>
<img src="/blog/images/bigdata/spark/spark_3_spark_r_new.png" width="450px">


<p>可以看出，SparkR 向量化是利用 Apache Arrow，其使得系统之间数据的交互变得很高效，而且避免了数据的序列化和反序列化的消耗，所以采用了这个之后，SparkR 和 Spark 交互的性能得到极大提升。</p>
<h2 id="9-Kafka-Streaming-includeHeaders"><a href="#9-Kafka-Streaming-includeHeaders" class="headerlink" title="9.Kafka Streaming: includeHeaders"></a>9.Kafka Streaming: includeHeaders</h2><p>Apache Kafka 0.11.0.0 版本支持在消息中配置一些 headers 信息，具体参见 KIP-82 - Add Record Headers，对应的 ISSUE 参见 KAFKA-4208。这些 Headers 在一些场景下很有用，Spark 3.0.0 为了满足用户的场景所以当然需要支持这个功能了，具体参见 SPARK-23539。具体使用如下</p>
<pre><code class="scala">
val df = spark 
            .readStream 
            .format(&quot;kafka&quot;) 
            .option(&quot;kafka.bootstrap.servers&quot;, &quot;host1:port1,host2:port2&quot;) 
            .option(&quot;subscribe&quot;, &quot;topic1&quot;)
            .option(&quot;includeHeaders&quot;, &quot;true&quot;)
            .load()

df.selectExpr(&quot;CAST(key AS STRING)&quot;, &quot;CAST(value AS STRING)&quot;, &quot;headers&quot;) .as[(String, String, Map)]</code></pre>
<h2 id="10-其他"><a href="#10-其他" class="headerlink" title="10.其他"></a>10.其他</h2><ul>
<li>Spark on K8S：Spark 对 Kubernetes 的支持是从2.3版本开始的，Spark 2.4 得到提升，Spark 3.0 将会加入 Kerberos 以及资源动态分配的支持。</li>
<li>Remote Shuffle Service：当前的 Shuffle 有很多问题，比如弹性差、对 NodeManager 有很大影响，不适应云环境。为了解决上面问题，将会引入 Remote Shuffle Service，具体参见 SPARK-25299</li>
<li>支持 JDK 11：参见 SPARK-24417，之所以直接选择 JDK 11 是因为 JDK 8 即将达到 EOL（end of life），而 JDK9 和 JDK10 已经是 EOL，所以社区就跳过 JDK9 和 JDK10 而直接支持 JDK11。不过 Spark 3.0 预览版默认还是使用 JDK 1.8；</li>
<li>移除对 Scala 2.11 的支持，默认支持 Scala 2.12，具体参见 SPARK-26132</li>
<li>支持 Hadoop 3.2，具体参见 SPARK-23710，Hadoop 3.0 已经发布了2年了（Apache Hadoop 3.0.0-beta1 正式发布，下一个版本(GA)即可在线上使用），所以支持 Hadoop 3.0 也是自然的，不过 Spark 3.0 预览版默认还是使用 Hadoop 2.7.4。</li>
<li>移除 Python 2.x 的支持：早在 2019年6月社区就有相关的讨论关于在 Spark 3.0 移除对 Python 2 的支持，目前 Spark 3.0.0 默认支持 Python 3.x ，参见 SPARK-27884。</li>
<li>Spark Graph 支持 Cypher：Cypher 是流行的图查询语言，现在我们可以直接在 Spark 3.0 使用 Cypher。</li>
<li>Spark event logs 支持 Roll 了，参见 《Spark 3.0 终于支持 event logs 滚动了》。</li>
</ul>
<h2 id="11-参考"><a href="#11-参考" class="headerlink" title="11.参考"></a>11.参考</h2><ul>
<li>[1] <a href="https://spark.apache.org/news/spark-3.0.0-preview.html" target="_blank" rel="noopener">Preview release of Spark 3.0</a></li>
<li>[2] <a href="https://spark.apache.org/news/spark-3.0.0-preview2.html" target="_blank" rel="noopener">Preview2 release of Spark 3.0</a></li>
<li>[3] <a href="https://www.mail-archive.com/dev@spark.apache.org/msg25781.html" target="_blank" rel="noopener">Apache Spark 3.0.0 RC1</a></li>
<li>[4] <a href="https://www.mail-archive.com/dev@spark.apache.org/msg26040.html" target="_blank" rel="noopener">pache Spark 3.0 RC2</a></li>
<li>[5] <a href="https://www.mail-archive.com/dev@spark.apache.org/msg26119.html" target="_blank" rel="noopener">Apache Spark 3.0 RC3</a></li>
<li>[6] <a href="https://spark.apache.org/releases/spark-release-3-0-0.html" target="_blank" rel="noopener">Spark Release 3.0.0</a></li>
<li>[7] <a href="https://issues.apache.org/jira/secure/attachment/12960252/SPIP_%20Accelerator-aware%20scheduling.pdf" target="_blank" rel="noopener">SPIP: Accelerator-aware scheduling</a></li>
</ul>
<h2 id="12-公众号"><a href="#12-公众号" class="headerlink" title="12.公众号"></a>12.公众号</h2><blockquote>
<p>本篇文章来自<strong>过往记忆</strong>大数据公众号，搞大数据的几乎都关注了。你还在犹豫？</p>
</blockquote>
<div class="pic_box">
    <div class="pic_parallel">
        <img src="/blog/images/bigdata/spark/guowangjiyi_wexin.jpg" width="200px">
        <span>微信公众号</span>
    </div>
        <div class="pic_parallel">
            <img src="/blog/images/bigdata/spark/guowangjiyi_bigdaa_ai.jpg" width="200px">
            <span>开发爱好者社区</span>
        </div>
</div>


      
       <hr><span style="font-style: italic;color: gray;"> 转载请注明来源，欢迎指出任何有错误或不够清晰的表达。可以在下面评论区评论，也可以邮件至 157162006@qq.com </span>
    </div>
</article>


<p>
    <a  class="dashang" onclick="dashangToggle()">赏</a>
</p>


<div class="article_copyright">
    <p><span class="copy-title">文章标题:</span>spark3.0新特性</p>
    <p><span class="copy-title">字数:</span><span class="post-count">4.3k</span></p>
    <p><span class="copy-title">本文作者:</span><a  title="沐雨云楼">沐雨云楼</a></p>
    <p><span class="copy-title">发布时间:</span>2020-06-21, 17:40:12</p>
    <p><span class="copy-title">最后更新:</span>2020-09-12, 21:21:47</p>
    <span class="copy-title">原始链接:</span><a class="post-url" href="/blog/2020/06/21/spark-version-3/" title="spark3.0新特性">https://iworkh.gitee.io/blog/2020/06/21/spark-version-3/</a>
    <p>
        <span class="copy-title">版权声明:</span><i class="fa fa-creative-commons"></i> <a rel="license noopener" href="http://creativecommons.org/licenses/by-nc-sa/4.0/" target="_blank" title="CC BY-NC-SA 4.0 International" target = "_blank">&#34;署名-非商用-相同方式共享 4.0&#34;</a> 转载请保留原文链接及作者。
    </p>
</div>



    <div id="comments"></div>
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/gitalk@1/dist/gitalk.css">

<script type="text/javascript">
    $.getScript('/blog/js/gitalk.js', function () {
        var gitalk = new Gitalk({
            clientID: 'e4a269e1ae1600031361',
            clientSecret: '6f3f981cbc49dd802fde779697d8f01da3a77981',
            repo: 'iworkh.github.io',
            owner: 'iworkh',
            admin: ['iworkh'],
            id: decodeURI(location.pathname),
            distractionFreeMode: 'true',
            language: 'zh-CN',
            perPage: parseInt('10',10)
        })
        gitalk.render('comments')
    })
</script>




    




    </div>
    <div class="copyright">
        <p class="footer-entry">©2020 iworkh</p>
<!--<p class="footer-entry">Built with <a href="https://hexo.io/" target="_blank">Hexo</a> and <a href="https://github.com/yelog/hexo-theme-3-hexo" target="_blank">3-hexo</a> theme</p>-->

    </div>
    <div class="full-toc">
        <button class="full" data-title="切换全屏 快捷键 s"><span class="min "></span></button>
<button class="post-toc-menu" data-title="打开TOC 快捷键 w"><span class="post-toc-menu-icons"></span></button>
<div class="post-toc"><span class="post-toc-title">目录</span>
    <div class="post-toc-content">

    </div>
</div>
<a class="" id="rocket" ></a>

    </div>
</div>
<div class="acParent"></div>

<div class="hide_box" onclick="dashangToggle()"></div>
<div class="shang_box">
    <a class="shang_close"  onclick="dashangToggle()">×</a>
    <div class="shang_tit">
        <p>喜欢就点赞,疼爱就打赏</p>
    </div>
    <div class="shang_payimg">
        <div class="pay_img">
            <img src="/blog/img/alipay.jpg" class="alipay" title="扫码支持">
            <img src="/blog/img/weixin.jpg" class="weixin" title="扫码支持">
        </div>
    </div>
    <div class="shang_payselect">
        <span><label><input type="radio" name="pay" checked value="alipay">支付宝</label></span><span><label><input type="radio" name="pay" value="weixin">微信</label></span>
    </div>
</div>


</body>
<script src="/blog/js/jquery.pjax.js?v=1.0.1" ></script>

<script src="/blog/js/script.js?v=1.0.1" ></script>
<script>
    var img_resize = 'default';
    /*作者、标签的自动补全*/
    $(function () {
        $('.search').AutoComplete({
            'data': ['@沐雨云楼','@iworkh','#ai','#attr','#aysnc','#beanmap','#bigdecimal','#blog','#chrome','#dict','#dll','#docker','#domain','#fastjson','#func','#git','#github','#hadoop','#hexo','#idea','#iworkh','#jackson','#java','#jdni','#jfinalshell','#joplin','#JPA','#lock','#markdowm','#maven','#mock','#mq','#mycat','#mysql','#nginx','#notebook','#plan','#powermock','#python','#redis','#rest','#sharding-jdbc','#spark','#springboot','#test','#tool','#tools','#tornado','#transactional','#web','#windows','#wtforms','#zookeeper','#分库分表','#分页','#命令','#工具','#并发','#序列化','#微服务','#性能分析','#手册','#有道云','#标签','#模板','#百度云盘','#结构和算法','#计划','#迁移','#面试',],
            'itemHeight': 20,
            'width': 418
        }).AutoComplete('show');
    })
    function initArticle() {
        /*渲染对应的表格样式*/
        
            $(".post .pjax table").addClass("green_title");
        

        /*渲染打赏样式*/
        
        $("input[name=pay]").on("click", function () {
            if($("input[name=pay]:checked").val()=="weixin"){
                $(".shang_box .shang_payimg .pay_img").addClass("weixin_img");
            } else {
                $(".shang_box .shang_payimg .pay_img").removeClass("weixin_img");
            }
        })
        

        /*高亮代码块行号*/
        
        $('pre code').each(function(){
            var lines = $(this).text().trim().split('\n').length, widther='';
            if (lines>99) {
                widther = 'widther'
            }
            var $numbering = $('<ul/>').addClass('pre-numbering ' + widther).attr("unselectable","on");
            $(this).addClass('has-numbering ' + widther)
                    .parent()
                    .append($numbering);
            for(var i=1;i<=lines;i++){
                $numbering.append($('<li/>').text(i));
            }
        });
        

        /*访问数量*/
        
        $.getScript("//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js");
        

        /*代码高亮，行号对齐*/
        $('.pre-numbering').css('line-height',$('.has-numbering').css('line-height'));

        
        
    }

    /*打赏页面隐藏与展示*/
    
    function dashangToggle() {
        $(".shang_box").fadeToggle();
        $(".hide_box").fadeToggle();
    }
    

</script>

<!--加入行号的高亮代码块样式-->

<style>
    pre{
        position: relative;
        margin-bottom: 24px;
        border-radius: 10px;
        border: 1px solid #e2dede;
        background: #FFF;
        overflow: hidden;
    }
    code.has-numbering{
        margin-left: 30px;
    }
    code.has-numbering.widther{
        margin-left: 35px;
    }
    .pre-numbering{
        margin: 0px;
        position: absolute;
        top: 0;
        left: 0;
        width: 20px;
        padding: 0.5em 3px 0.7em 5px;
        border-right: 1px solid #C3CCD0;
        text-align: right;
        color: #AAA;
        background-color: #fafafa;
    }
    .pre-numbering.widther {
        width: 35px;
    }
</style>

<!--自定义样式设置-->
<style>
    
    
    .nav {
        width: 442px;
    }
    .nav.fullscreen {
        margin-left: -442px;
    }
    .nav-left {
        width: 120px;
    }
    
    
    @media screen and (max-width: 1468px) {
        .nav {
            width: 442px;
        }
        .nav.fullscreen {
            margin-left: -442px;
        }
        .nav-left {
            width: 150px;
        }
    }
    
    
    @media screen and (max-width: 1024px) {
        .nav {
            width: 442px;
            margin-left: -442px
        }
        .nav.fullscreen {
            margin-left: 0;
        }
        .nav .hide-list.fullscreen {
            left: 442px
        }
    }
    
    @media screen and (max-width: 426px) {
        .nav {
            width: 100%;
        }
        .nav-left {
            width: 100%;
        }
    }
    
    
    .nav-right .title-list nav a .post-title, .nav-right .title-list #local-search-result a .post-title {
        color: #383636;
    }
    
    
    .nav-right .title-list nav a .post-date, .nav-right .title-list #local-search-result a .post-date {
        color: #5e5e5f;
    }
    
    
    .nav-right nav a.hover, #local-search-result a.hover{
        background-color: #e2e0e0;
    }
    
    

    /*列表样式*/
    
    .post .pjax article .article-entry>ol, .post .pjax article .article-entry>ul, .post .pjax article>ol, .post .pjax article>ul{
        border: #e2dede solid 1px;
        border-radius: 10px;
        padding: 10px 32px 10px 56px;
    }
    .post .pjax article .article-entry li>ol, .post .pjax article .article-entry li>ul,.post .pjax article li>ol, .post .pjax article li>ul{
        padding-top: 5px;
        padding-bottom: 5px;
    }
    .post .pjax article .article-entry>ol>li, .post .pjax article .article-entry>ul>li,.post .pjax article>ol>li, .post .pjax article>ul>li{
        margin-bottom: auto;
        margin-left: auto;
    }
    .post .pjax article .article-entry li>ol>li, .post .pjax article .article-entry li>ul>li,.post .pjax article li>ol>li, .post .pjax article li>ul>li{
        margin-bottom: auto;
        margin-left: auto;
    }
    

    /* 背景图样式 */
    
    


    /*引用块样式*/
    

    /*文章列表背景图*/
    

    
</style>






<div class="mobile-menus-out" >

</div>
<div class="mobile-menus">
    
    
    
    <a class="dynamic-menu " target="_blank"  href="https://pgmanor.gitee.io/blog/">pgmanor</a>
    
    
    <a class="dynamic-menu " target="_blank"  href="https://www.iworkh.com/manualIt/Category/scopeDev/">iworkh</a>
    
    
    <a class="dynamic-menu " target="_blank"  href="https://gitee.com/iworkh">gitee</a>
    
</div>


<div style="position:absolute; bottom: 0; right: 0;">
    <iframe frameborder="no" border="0" marginwidth="0" marginheight="0" width=150 height=40 src="//music.163.com/outchain/player?type=2&id=1323150930&auto=0&height=32"></iframe>
</div>

</html>
